{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_content = os.listdir()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_content = [filename for filename in dir_content if filename.find('enduro') > -1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1524545416-enduro-v0.csv', '1524545592-enduro-v0.csv']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dir_content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(dir_content[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>actions-input</th>\n",
       "      <th>rewards</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0, actions-input, rewards]\n",
       "Index: []"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df.rewards > 0.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "first_sequence = df['actions-input'][:316]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[8, 6, 7, 2],\n",
       " [0, 4, 3, 8],\n",
       " [4, 7, 6, 5],\n",
       " [1, 5, 1, 4],\n",
       " [2, 5, 8, 4],\n",
       " [0, 7, 3, 6],\n",
       " [3, 1, 3, 3],\n",
       " [7, 8, 6, 8],\n",
       " [4, 3, 5, 3],\n",
       " [0, 5, 2, 1],\n",
       " [3, 8, 6, 3]]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_sequence = first_sequence.tolist()\n",
    "treated_sequence = []\n",
    "start = 0\n",
    "incremental= 4\n",
    "limit = 4\n",
    "reversed_sequence = list_sequence[::-1]\n",
    "#reversed_sequence[start:limit]\n",
    "#reversed_sequence[limit+4:]\n",
    "reversed_sequence_len = lambda: range(len(reversed_sequence))\n",
    "for _ in reversed_sequence_len():\n",
    "    done = not reversed_sequence[start:limit]\n",
    "    if not done: treated_sequence.append(reversed_sequence[start:limit])\n",
    "    reversed_sequence = reversed_sequence[limit+incremental:]\n",
    "    start = start + incremental\n",
    "    limit = limit + incremental\n",
    "    if done: break\n",
    "treated_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Produced length of output: 11\n"
     ]
    }
   ],
   "source": [
    "print('Produced length of output: {}'.format(len(treated_sequence)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "actions = np.array(treated_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[8, 6, 7, 2],\n",
       "       [0, 4, 3, 8],\n",
       "       [4, 7, 6, 5],\n",
       "       [1, 5, 1, 4],\n",
       "       [2, 5, 8, 4],\n",
       "       [0, 7, 3, 6],\n",
       "       [3, 1, 3, 3],\n",
       "       [7, 8, 6, 8],\n",
       "       [4, 3, 5, 3],\n",
       "       [0, 5, 2, 1],\n",
       "       [3, 8, 6, 3]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "second_sequence = df['rewards'][:316]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0],\n",
       " [0.0, 0.0, 0.0, 0.0]]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_sequence = second_sequence.tolist()\n",
    "treated_sequence = []\n",
    "start = 0\n",
    "incremental = 4\n",
    "limit = 4\n",
    "reversed_sequence = list_sequence[::-1]\n",
    "reversed_sequence_len = lambda: range(len(reversed_sequence))\n",
    "for _ in reversed_sequence_len():\n",
    "    done = not reversed_sequence[start:limit]\n",
    "    if not done: treated_sequence.append(reversed_sequence[start:limit])\n",
    "    reversed_sequence = reversed_sequence[limit+incremental:]\n",
    "    start = start + incremental\n",
    "    limit = limit + incremental\n",
    "    if done: break\n",
    "treated_sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Produced length of output: 11\n"
     ]
    }
   ],
   "source": [
    "print('Produced length of output: {}'.format(len(treated_sequence)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "rewards = np.array(treated_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings; warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import model_selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "x, x_val, y, y_val = model_selection.train_test_split(rewards, actions, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.add(Dense(4, input_shape=(4,)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='rmsprop', loss='mse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8 samples, validate on 3 samples\n",
      "Epoch 1/10\n",
      "8/8 [==============================] - 0s 13ms/step - loss: 24.5312 - val_loss: 23.0586\n",
      "Epoch 2/10\n",
      "8/8 [==============================] - 0s 274us/step - loss: 24.5034 - val_loss: 23.0406\n",
      "Epoch 3/10\n",
      "8/8 [==============================] - 0s 305us/step - loss: 24.4832 - val_loss: 23.0256\n",
      "Epoch 4/10\n",
      "8/8 [==============================] - 0s 304us/step - loss: 24.4663 - val_loss: 23.0123\n",
      "Epoch 5/10\n",
      "8/8 [==============================] - 0s 326us/step - loss: 24.4513 - val_loss: 23.0001\n",
      "Epoch 6/10\n",
      "8/8 [==============================] - 0s 334us/step - loss: 24.4376 - val_loss: 22.9887\n",
      "Epoch 7/10\n",
      "8/8 [==============================] - 0s 292us/step - loss: 24.4248 - val_loss: 22.9779\n",
      "Epoch 8/10\n",
      "8/8 [==============================] - 0s 352us/step - loss: 24.4126 - val_loss: 22.9676\n",
      "Epoch 9/10\n",
      "8/8 [==============================] - 0s 320us/step - loss: 24.4010 - val_loss: 22.9576\n",
      "Epoch 10/10\n",
      "8/8 [==============================] - 0s 298us/step - loss: 24.3898 - val_loss: 22.9480\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x7fa5e596cbe0>"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x, y, epochs=10, batch_size=32, validation_data=(x_val, y_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8/8 [==============================] - 0s 187us/step\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(x, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24.378910064697266"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 3.361495  , -5.505101  , -2.6514647 , -0.34251547],\n",
       "       [ 4.267953  , -3.1263242 , -3.0491097 , -0.05232491]],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(np.array([[2,5,1,0],[1,1,4,2]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python ../app.py --environment_name \"Pong-v0\" --output_stats_filename \"pong-v0\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
